package util

import (
	"ChinaDistrictCrawler/internal/model/entity"
	"ChinaDistrictCrawler/internal/service"
	"context"
	"fmt"
	"github.com/antchfx/htmlquery"
	"github.com/gogf/gf/v2/util/gconv"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

const (
	baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/index.html"
)

var urlRegEx = regexp.MustCompile("^.+/([\\d]+)\\.html$")

func FetchAll(ctx context.Context) {
	provinces, pUrls, err := GetProvinceUrls()
	if err != nil {
		fmt.Println(err)
		return
	}
	countP := len(provinces)
	for pi := 0; pi < countP; pi++ {
		_, err = service.Province().Create(ctx, provinces[pi])
		if err != nil {
			fmt.Println(err)
			return
		}
		cities, cUrls, err := GetCityUrls(pUrls[pi], &provinces[pi])
		countC := len(cities)
		for ci := 0; ci < countC; ci++ {
			_, err = service.City().Create(ctx, cities[ci])
			if err != nil {
				fmt.Println(err)
				return
			}
			districts, dUrls, err := GetDistrictUrls(cUrls[ci], &cities[ci])
			countD := len(districts)
			for di := 0; di < countD; di++ {
				_, err = service.District().Create(ctx, districts[di])
				if err != nil {
					fmt.Println(err)
					return
				}
				streets, sUrls, err := GetStreetUrls(dUrls[di], &districts[di])
				countS := len(streets)
				for si := 0; si < countS; si++ {
					_, err = service.Street().Create(ctx, streets[si])
					if err != nil {
						fmt.Println(err)
						return
					}
					residents, err := GetResidentUrls(sUrls[si], &streets[si])
					countR := len(residents)
					for ri := 0; ri < countR; ri++ {
						_, err = service.Resident().Create(ctx, residents[ri])
						if err != nil {
							fmt.Println(err)
							return
						}
					}
				}
			}
		}
	}
}

func GetProvinceUrls() ([]entity.Province, []string, error) {
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, _ := http.NewRequest("GET", baseUrl, nil)
	resp, err := client.Do(req)
	if err != nil {
		return nil, []string{}, err
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			fmt.Println("close error", err)
		}
	}(resp.Body)

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("read error", err)
		return nil, []string{}, err
	}
	doc, _ := htmlquery.Parse(strings.NewReader(string(body)))

	links := htmlquery.Find(doc, "//tr[@class=\"provincetr\"]/td/a")

	count := len(links)
	var provinces = make([]entity.Province, count)
	var pUrls = make([]string, count)
	for i, link := range links {
		pUrl := absUrl(link.Attr[0].Val, baseUrl)
		name := htmlquery.InnerText(link)
		id := urlRegEx.FindStringSubmatch(pUrl)[1]
		provinces[i] = entity.Province{
			ProvinceId:   gconv.Int(id),
			ProvinceCode: id + "0000000000",
			ProvinceName: name,
		}
		pUrls[i] = pUrl
	}
	fmt.Printf("抓取 %s 得到省份 %d 个\n", baseUrl, count)
	return provinces, pUrls, nil
}

func GetCityUrls(url string, province *entity.Province) ([]entity.City, []string, error) {
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(req)
	if err != nil {
		return nil, []string{}, err
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			fmt.Println("close error", err)
		}
	}(resp.Body)

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("read error", err)
		return nil, []string{}, err
	}
	doc, _ := htmlquery.Parse(strings.NewReader(string(body)))

	links := htmlquery.Find(doc, "//tr[@class=\"citytr\"]/td/a")

	count := len(links)
	var cities = make([]entity.City, count/2)
	var urls = make([]string, count/2)
	for i := 0; i < count; i += 2 {
		cUrl := absUrl(links[i].Attr[0].Val, url)
		code := htmlquery.InnerText(links[i])
		name := htmlquery.InnerText(links[i+1])
		id := urlRegEx.FindStringSubmatch(cUrl)[1]
		cities[i/2] = entity.City{
			CityId:       gconv.Int(id),
			CityCode:     code,
			CityName:     name,
			ProvinceId:   province.ProvinceId,
			ProvinceCode: province.ProvinceCode,
			ProvinceName: province.ProvinceName,
		}
		urls[i/2] = cUrl
	}
	fmt.Printf("抓取省份 %s %s 得到地市 %d 个\n", province.ProvinceName, url, count/2)
	return cities, urls, nil
}

func GetDistrictUrls(url string, city *entity.City) ([]entity.District, []string, error) {
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(req)
	if err != nil {
		return nil, []string{}, err
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			fmt.Println("close error", err)
		}
	}(resp.Body)

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("read error", err)
		return nil, []string{}, err
	}
	doc, _ := htmlquery.Parse(strings.NewReader(string(body)))

	links := htmlquery.Find(doc, "//tr[@class=\"countytr\"]/td/a")

	count := len(links)
	var districts = make([]entity.District, count/2)
	var urls = make([]string, count/2)
	for i := 0; i < count; i += 2 {
		dUrl := absUrl(links[i].Attr[0].Val, url)
		code := htmlquery.InnerText(links[i])
		name := htmlquery.InnerText(links[i+1])
		id := urlRegEx.FindStringSubmatch(dUrl)[1]
		districts[i/2] = entity.District{
			DistrictId:   gconv.Int(id),
			DistrictCode: code,
			DistrictName: name,
			ProvinceId:   city.ProvinceId,
			ProvinceCode: city.ProvinceCode,
			ProvinceName: city.ProvinceName,
			CityId:       city.CityId,
			CityCode:     city.CityCode,
			CityName:     city.CityName,
		}
		urls[i/2] = dUrl
	}
	fmt.Printf("抓取地市 %s %s 得到区县 %d 个\n", city.ProvinceName+city.CityName, url, count/2)
	return districts, urls, nil
}

func GetStreetUrls(url string, district *entity.District) ([]entity.Street, []string, error) {
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(req)
	if err != nil {
		return nil, []string{}, err
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			fmt.Println("close error", err)
		}
	}(resp.Body)

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("read error", err)
		return nil, []string{}, err
	}
	doc, _ := htmlquery.Parse(strings.NewReader(string(body)))

	links := htmlquery.Find(doc, "//tr[@class=\"towntr\"]/td/a")

	count := len(links)
	var streets = make([]entity.Street, count/2)
	var urls = make([]string, count/2)
	for i := 0; i < count; i += 2 {
		sUrl := absUrl(links[i].Attr[0].Val, url)
		code := htmlquery.InnerText(links[i])
		name := htmlquery.InnerText(links[i+1])
		id := urlRegEx.FindStringSubmatch(sUrl)[1]
		streets[i/2] = entity.Street{
			StreetId:     gconv.Int(id),
			StreetCode:   code,
			StreetName:   name,
			ProvinceId:   district.ProvinceId,
			ProvinceCode: district.ProvinceCode,
			ProvinceName: district.ProvinceName,
			CityId:       district.CityId,
			CityCode:     district.CityCode,
			CityName:     district.CityName,
			DistrictId:   district.DistrictId,
			DistrictCode: district.DistrictCode,
			DistrictName: district.DistrictName,
		}
		urls[i/2] = sUrl
	}
	fmt.Printf("抓取区县 %s %s 得到街道 %d 个\n", district.ProvinceName+district.CityName+district.DistrictName, url, count/2)
	return streets, urls, nil
}

func GetResidentUrls(url string, street *entity.Street) ([]entity.Resident, error) {
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			fmt.Println("close error", err)
		}
	}(resp.Body)

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("read error", err)
		return nil, err
	}
	doc, _ := htmlquery.Parse(strings.NewReader(string(body)))

	links := htmlquery.Find(doc, "//tr[@class=\"villagetr\"]/td")

	count := len(links)
	var residents = make([]entity.Resident, count/3)
	for i := 0; i < count; i += 3 {
		code := htmlquery.InnerText(links[i])
		category := htmlquery.InnerText(links[i+1])
		name := htmlquery.InnerText(links[i+2])
		residents[i/3] = entity.Resident{
			ResidentId:       gconv.Int64(code),
			ResidentCategory: gconv.Int(category),
			ResidentCode:     code,
			ResidentName:     name,
			ProvinceId:       street.ProvinceId,
			ProvinceCode:     street.ProvinceCode,
			ProvinceName:     street.ProvinceName,
			CityId:           street.CityId,
			CityCode:         street.CityCode,
			CityName:         street.CityName,
			DistrictId:       street.DistrictId,
			DistrictCode:     street.DistrictCode,
			DistrictName:     street.DistrictName,
			StreetId:         street.StreetId,
			StreetCode:       street.StreetCode,
			StreetName:       street.StreetName,
		}
	}
	fmt.Printf("抓取街道 %s %s 得到社区 %d 个\n", street.ProvinceName+street.CityName+street.DistrictName+street.StreetName, url, count/3)
	return residents, nil
}

// 返回url的绝对地址
// currUrl 当前url
func absUrl(currUrl, baseUrl string) string {
	urlInfo, err := url.Parse(currUrl)
	if err != nil {
		return ""
	}
	if urlInfo.Scheme != "" {
		return currUrl
	}
	baseInfo, err := url.Parse(baseUrl)
	if err != nil {
		return ""
	}

	u := baseInfo.Scheme + "://" + baseInfo.Host
	var path string
	if strings.Index(urlInfo.Path, "/") == 0 {
		path = urlInfo.Path
	} else {
		path = filepath.Dir(baseInfo.Path) + "/" + urlInfo.Path
	}

	rst := make([]string, 0)
	pathArr := strings.Split(path, "/")

	// 如果path是已/开头，那在rst加入一个空元素
	if pathArr[0] == "" {
		rst = append(rst, "")
	}
	for _, p := range pathArr {
		if p == ".." {
			if rst[len(rst)-1] == ".." {
				rst = append(rst, "..")
			} else {
				rst = rst[:len(rst)-1]
			}
		} else if p != "" && p != "." {
			rst = append(rst, p)
		}
	}
	return u + strings.Join(rst, "/")
}
